from gxl_ai_utils.utils import utils_file

# url = "https://huggingface.co/datasets/BAAI/CCI2-Data/resolve/main/data/cci2-00177-of-00178.parquet?download=true"
# utils_file.download_file(url, target_dir='./data/', suffix='parquet')
num = 178
target_dir = '/home/work_nfs8/xlgeng/data/BAAI_CCI_2_data/'
for i in range(num):
    i = str(i).zfill(5)
    url = f"https://huggingface.co/datasets/BAAI/CCI2-Data/resolve/main/data/cci2-{i}-of-00178.parquet?download=true"
    utils_file.download_file(url, target_dir=target_dir, suffix='parquet')
